from scrapy import Request
import json
import datetime
import scrapy
from scrapyspider.items import Sort

# 用于爬取赛氪网中，比赛与比赛类别的关系
#   request.xpath('//a[contains(@class,"fl item-link")]/@href').extract()
#   ['/vs/mcm/0/0', '/vs/acm/0/0', '/vs/robot/0/0', '/vs/machine/0/0', '/vs/building/0/0', '/vs/big_data/0/0', '/vs/traffic/0/0', '/vs/airplane/0/0', '/vs/ship_sea/0/0', '/vs/energy/0/0', '/vs/computer/0/0', '/vs/material/0/0', '/vs/electronic/0
# /0', '/vs/industry_design/0/0', '/vs/english/0/0', '/vs/speech/0/0', '/vs/model/0/0', '/vs/song_and_dance/0/0', '/vs/sports/0/0', '/vs/technology/0/0', '/vs/ui_design/0/0', '/vs/fashion_design/0/0', '/vs/electronic_games/0/0', '/vs/match/0/0
# ', '/vs/physics/0/0', '/vs/chemistry/0/0', '/vs/health/0/0', '/vs/mechanics/0/0', '/vs/occupation/0/0', '/vs/tiaozhanbei/0/0', '/vs/environment/0/0', '/vs/society/0/0', '/vs/chuangye/0/0', '/vs/business/0/0', '/vs/chuangqingchun/0/0']
#
#


class SaikrSpider(scrapy.Spider):
    name = 'sort_saikr'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36',
    }

    def start_requests(self):
        types = ['/vs/mcm/0/0', '/vs/acm/0/0', '/vs/robot/0/0', '/vs/machine/0/0', '/vs/building/0/0', '/vs/big_data/0/0', '/vs/traffic/0/0', '/vs/airplane/0/0', '/vs/ship_sea/0/0', '/vs/energy/0/0', '/vs/computer/0/0', '/vs/material/0/0', '/vs/electronic/0 /0', '/vs/industry_design/0/0', '/vs/english/0/0', '/vs/speech/0/0', '/vs/model/0/0', '/vs/song_and_dance/0/0', '/vs/sports/0/0', '/vs/technology/0/0', '/vs/ui_design/0/0', '/vs/fashion_design/0/0', '/vs/electronic_games/0/0', '/vs/match/0/0 ', '/vs/physics/0/0', '/vs/chemistry/0/0', '/vs/health/0/0', '/vs/mechanics/0/0', '/vs/occupation/0/0', '/vs/tiaozhanbei/0/0', '/vs/environment/0/0', '/vs/society/0/0', '/vs/chuangye/0/0', '/vs/business/0/0', '/vs/chuangqingchun/0/0']
        for i in types:
            url = 'https://www.saikr.com{}'.format(i)
            yield Request(url, headers=self.headers)

    def parse(self, response):
        contests = response.xpath('//li[@class="item clearfix"]')
        t = response.xpath('//a[contains(@class,"fl item-link active")]/text()').extract()[0].strip()
        for li in contests:
            sort = Sort()
            level = li.xpath('div//p/text()').extract()[1]
            sort['contest_name'] =li.xpath('div/h3[@class="tit"]/a/text()').extract()[0].strip()
            sort['type_name'] = t
            yield sort

        next_url =response.xpath('//li[@class="next"]/a/@href').extract()
        if( next_url):
            next_url = 'https://www.saikr.com' + next_url[0].strip()
            yield Request(next_url, headers=self.headers)







